#!/usr/bin/env python
# -*- encoding: utf-8 -*-
# Created on 2016-05-21 20:21:32
# Project: tutorial_douban_moive

from pyspider.libs.base_handler import *
import re

class Handler(BaseHandler):
    crawl_config = {
    }

    @every(minutes=24 * 60)
    def on_start(self):
        url = "http://movie.douban.com/tag/"
        self.crawl(url, callback=self.index_page, validate_cert = False)

    @config(age=10 * 24 * 60 * 60)
    def index_page(self, response):
        for each in response.doc('a[href^="http"]').items():
            if re.match("https://movie.douban.com/tag/\w+", each.attr.href, re.U):
                self.crawl(each.attr.href, callback=self.list_page, validate_cert = False)

    def list_page(self, response):
        for each in response.doc('#content > div > div.article > div > table  tr > td > div > a').items():
            #print each
            self.crawl(each.attr.href, callback=self.detail_page, validate_cert = False)

    @config(priority=2)
    def detail_page(self, response):
        return {
            "url": response.url,
            "title": response.doc('#content > h1 > span').text(),
            "rating": response.doc('#interest_sectl > div.rating_wrap.clearbox > div.rating_self.clearfix > strong').text(),
            "director": [x.text() for x in response.doc('#info > span:nth-child(1) > span.attrs > a').items()],
        }